# encoding: utf-8

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import re

class UnmatchedSequenceError(Exception):
    def __init__(self, text):
        self.text = text


class Token(object):
    def __init__(self, type, text=None):
        self.type = type
        if text is not None:
            self.text = text
        else:
            self.text = type


# rules = [('pattern', token_iterable callback(regex_match)), ... ]
class Scanner(object):
    def __init__(self, rules):
        self._rules = []
        for rule in rules:
            self._rules.append((re.compile(rule[0]), rule[1]))

    def scan(self, text):
        begin = 0
        while begin < len(text):
            for rule in self._rules:
                m = rule[0].match(text, begin)
                if m is not None:
                    for token in rule[1](m):
                        yield token
                    begin = m.end()
                    break
            else:
                raise UnmatchedSequenceError(text[begin:])

